#!/usr/bin/env python
#:usage: python lazed.py main /opt/logfile/chn/main/ftptemp
import os,sys,re
import fnmatch
import time

#===globe counters===
logstatic={"readlines":0,"droplines":0,"all_clients":0,"all_hosts":0,"time":time.time(),"all_sc":0.0,"all_cs":0.0}

g_clients={}
#"client-ip":[req_counts,denied_req_counts,scbytes,csbytes,{"denihost-ip":[req_counts,0,0,0]}
g_hosts={}
#"host-ip":[req_counts,denied_req_counts,scbytes,csbytes,{}]

g_days={}
# "2011-09-08":[req_counts,p,scbytes,csbytes]

all_log_structs={"MAIN":{"client-ip":3,"host-ip":15,"sc":22,"cs":23}, # ref ash810 main
              "CIFS":{"client-ip":2,"host-ip":4,"sc":24,"cs":23,"grepword":"SUCCESS"},
              "SSL":{"client-ip":3,"host-ip":14,"sc":19,"cs":20}
             }

def dic_topten(dic,f):
    """
    return a list[("--Empty--",[0,0,0,0]),]
    """
    top_s=sorted(dic.iteritems(),key=lambda x:x[1][f],reverse=True)
    if len(top_s)>=10:
        return top_s[0:10]
    else:
        return top_s

def list_update(alist,r,dr,sc,cs):
	alist[0]+=r
	alist[1]+=dr
	alist[2]+=sc
	alist[3]+=cs
        logstatic["all_sc"]+=sc
        logstatic["all_cs"]+=cs

def dic_update(dic,key,sc=0,cs=0,dst=None): #
	if dst is not None: #update a denied record
		if key not in dic: dic[key]=[1,1,sc,cs,{dst:[1,0,0,0]}]
		elif dst not in dic[key][4]:
			list_update(dic[key],1,1,sc,cs)
			dic[key][4][dst]=[1,1,0,0]
		else:
			list_update(dic[key],1,0,sc,cs)
			list_update(dic[key][4][dst],1,0,0,0)
	else:
		if key not in dic: dic[key]=[1,0,sc,cs,{}]
		else: list_update(dic[key],1,0,sc,cs)

def killspacebyline(line):
	reg = re.compile(r'".*?"')
        for qued in re.findall(reg,line):
            r=qued.replace(" ","_")
            line=line.replace(qued,r)
        return line

def filterline(line,log_struct):
	if line[0] is  "#":
		logstatic["droplines"]+=1
		return -1
	if "grepword" in log_struct:
		if log_struct["grepword"] not in line:
			logstatic["droplines"]+=1
			return -1

	k=killspacebyline(line).split()
	if "-" == k[log_struct["sc"]] or "-" == k[log_struct["cs"]]:
		logstatic["droplines"]+=1
		return -1

	if "-" == k[log_struct["host-ip"]] or "-" == k[log_struct["client-ip"]]:
		logstatic["droplines"]+=1
		return -1
	if "policy_denied" in line: return (1,k) #with dst
	else: return (0,k) #without dst

def coroute(func):
            def wrapper(*args,**kw):
                gen = func(*args, **kw)
                gen.next()
                return gen
            wrapper.__name__ = func.__name__
            wrapper.__dict__ = func.__dict__
            wrapper.__doc__  = func.__doc__
            return wrapper
@coroute
def find_files(tg):
	while True:
		topdir,pattern = (yield)
		for path,dirname,filelist in os.walk(topdir):
			for name in filelist:
				if fnmatch.fnmatch(name,pattern):
					print "<!-- #Reading file:",os.path.join(path,name),"-->"
					tg.send(os.path.join(path,name))

import gzip

@coroute
def opener(tg):
	while True:
		name = (yield)
		if name.endswith(".gz"): f = gzip.open(name)
		elif name.endswith(".done"): f = gzip.open(name)
		else: f = open(name)
		tg.send(f)

@coroute
def catlines(tg):
	while True:
		f = (yield)
		for line in f:
			tg.send(line)

@coroute
def procline(log_struct):
	while True:
		line  = (yield)
		logstatic["readlines"]+=1
		f_res=filterline(line,log_struct)
		if type(f_res) is tuple:
			fds=f_res[1]
                        dic_update(g_hosts,fds[log_struct["host-ip"]],int(fds[log_struct["sc"]]),
						int(fds[log_struct["cs"]]))
                        dic_update(g_days,fds[0],int(fds[log_struct["sc"]]),
						int(fds[log_struct["cs"]]))
			if f_res[0] == 1: #dic_update(dic,key,sc=0,cs=0,dst=not_None)
                            dic_update(g_clients,fds[log_struct["client-ip"]],
						int(fds[log_struct["sc"]]),
						int(fds[log_struct["cs"]]),fds[log_struct["host-ip"]])
			elif f_res[0] == 0:#dst=None
                            dic_update(g_clients,fds[log_struct["client-ip"]],
							int(fds[log_struct["sc"]]),
							int(fds[log_struct["cs"]]))
			else: pass

def getDenied(k):
    dic=g_clients[k][4]
    output=""
    for h in dic_topten(dic,0):
        output+="{0} [Denied:{1} times]\n".format(\
                h[0],h[1][1])
    return output

def humanRead(n):
    if n < 1000000:
        return "<1MB"
    if n < 1000000000 and n >1000000:
        return "{0:.2f}MB".format(n/1000000.00)
    else:
        return "{0:.2f}GB".format(n/1000000000.00)
def drawBar(i,t):
    percent=i*100.00/t
    bar="["
    for b in range(int(percent/5)):
         bar+=">>"
    bar+="]{0:.2f}%"
    return bar.format(percent)

def run():
    log_type=sys.argv[1].upper()
    log_struct=all_log_structs[log_type]
    pyzcater = find_files(opener(catlines(procline(log_struct))))
    pyzcater.send((sys.argv[2],"*gz*"))

    logstatic["all_clients"]=len(g_clients)
    logstatic["all_hosts"]=len(g_hosts)
    logstatic["time"]=time.time()-logstatic["time"]
    total_bytes=logstatic["all_sc"]+logstatic["all_cs"]


    #############Out put#################
    print """
            <div id="rephead">
            <a id="{reportname}"></a>
            <br>
            <big>#Section of {reportname}:</big></br>
            <a href="#CIFS">CIFS</a>   <a href="#MAIN">Main</a>    <a href="#SSL">SSL</a></br>
            </div>
          """.format(reportname=log_type)

    print "<!--Weekly Static -->"
    print """<br><div id="whead">
            Total Requests: {0}<br>
            Total Bytes(In+Out): {1}<br>
            Total Clients' IP: {2}<br>
            Total Hosts' IP: {3}<br>
            </div>\n
            <!--HIDE_total_bytes {4}:{5} -->\n""".format(\
                logstatic["readlines"],\
                humanRead(total_bytes),\
                logstatic["all_clients"],\
                logstatic["all_hosts"],\
                log_type,\
                humanRead(total_bytes))

    print "<!-- Daily Usage -->"
    print """
          <br><table id="custab1">
          <caption>{0} - Daily Usage</caption>
          <tr>
            <th>Date</th>
            <th colspan="2">Request</th>
            <th>BytesIn</th>
            <th>BytesOut</th>
            <th colspan="2">Total bytes</th>
         </tr>""".format(log_type)

    for day in sorted(g_days):
        print """
              <tr>
                  <td>{0}</td>
                  <td>{1}</td>
                  <td>{2:0.2f}%</td>
                  <td>{3}</td>
                  <td>{4}</td>
                  <td>{5}</td>
                  <td>{6}</td>
              </tr>
              """.format(\
                      day,
                      g_days[day][0],
                      g_days[day][0]/logstatic["readlines"],
                      humanRead(g_days[day][2]),
                      humanRead(g_days[day][3]),
                      humanRead(g_days[day][3] + g_days[day][2]),
                      drawBar(g_days[day][2]+g_days[day][3],total_bytes))
    print "</table><!-- end of daily usage -->"

    print "<!--     Top 10 of Hosts By Requests -->"
    print """
            <br><table id="custab2">
            <caption>{0} - Top 10 of Hosts By Requests</caption>
                <tr>
                    <th>Host</th>
                    <th>BytesIn</th>
                    <th>BytesOut</th>
                    <th colspan="2">Requests</th>
                </tr>
          """.format(log_type)
    for i in dic_topten(g_hosts,0):
        print """
              <tr>
                <td>{0}</td>
                <td>{1}</td>
                <td>{2}</td>
                <td>{3}</td>
                <td>{4}</td>
             </tr>
             """.format(\
                     i[0],\
                     humanRead(i[1][2]),\
                     humanRead(i[1][3]),\
                     i[1][0],\
                     drawBar(i[1][0],logstatic["readlines"]))
    print "</table><!-- end of Top 10 of Hosts By Requests -->"

    print "<!-- Top 10 of Hosts By BytesIn -->"
    print """
            <br><table id="custab2">
            <caption>{0} - Top 10 of Hosts By BytesIn</caption>
                <tr>
                    <th>Host</th>
                    <th>Requests</th>
                    <th>BytesOut</th>
                    <th colspan="2">BytesIn</th>
                </tr>
          """.format(log_type)
    for i in dic_topten(g_hosts,2):
        print """
              <tr>
                <td>{0}</td>
                <td>{1}</td>
                <td>{2}</td>
                <td>{3}</td>
                <td>{4}</td>
             </tr>
             """.format(\
                     i[0],\
                     i[1][0],\
                     humanRead(i[1][3]),\
                     humanRead(i[1][2]),\
                     drawBar(i[1][2],logstatic["all_sc"]))
    print "</table><!-- end of  Top 10 of Hosts By BytesIn -->"

    print "<br/>"

    print "<!--     Top 10 of Clients By Requests -->"
    print """
            <br><table id="custab3">
            <caption>{0} - Top 10 of Clients By Requests</caption>
                <tr>
                    <th>Client</th>
                    <th>BytesIn</th>
                    <th>BytesOut</th>
                    <th colspan="2">Requests</th>
                </tr>
          """.format(log_type)
    for i in dic_topten(g_clients,0):
        print """
              <tr>
                <td>{0}</td>
                <td>{1}</td>
                <td>{2}</td>
                <td>{3}</td>
                <td>{4}</td>
             </tr>
             """.format(\
                     i[0],\
                     humanRead(i[1][2]),\
                     humanRead(i[1][3]),\
                     i[1][0],\
                     drawBar(i[1][0],logstatic["readlines"]))
    print "</table><!-- end of Top 10 of Clients By Requests -->"

    print "<!-- Top 10 of Clients By BytesIn -->"
    print """
            <br><table id="custab3">
            <caption>{0} - Top 10 of Clients By BytesIn</caption>
                <tr>
                    <th>Clients</th>
                    <th>Requests</th>
                    <th>BytesOut</th>
                    <th colspan="2">BytesIn</th>
                </tr>
          """.format(log_type)
    for i in dic_topten(g_clients,2):
        print """
              <tr>
                <td>{0}</td>
                <td>{1}</td>
                <td>{2}</td>
                <td>{3}</td>
                <td>{4}</td>
             </tr>
             """.format(\
                     i[0],\
                     i[1][0],\
                     humanRead(i[1][3]),\
                     humanRead(i[1][2]),\
                     drawBar(i[1][2],logstatic["all_sc"]))
    print "</table><!-- end of  Top 10 of Clients By BytesIn -->"

    print "<!--     Top 10 of Clients By Denied Requests -->"
    print """
            <br><table id="custab3">
            <caption>{0} - Top 10 of Clients Denied Requests</caption>
                <tr>
                    <th>Client</th>
                    <th>Requests</th>
                    <th>Denied Requests</th>
                    <th>Most Denied</th>
                </tr>
          """.format(log_type)
    for i in dic_topten(g_clients,1):
        print """
              <tr>
                <td>{0}</td>
                <td>{1}</td>
                <td>{2}</td>
                <th><pre>{3}</pre></th>
             </tr>
             """.format(\
                     i[0],\
                     i[1][0],\
                     i[1][1],\
                     getDenied(i[0]))
    print "</table><!-- end of Top 10 of Clients By Requests -->"

if __name__ == "__main__":
    run()
